package com.sentiment.crawler;

import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;

import org.apache.log4j.PropertyConfigurator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.sentiment.config.Config;
import com.sentiment.config.ConfigCrawler;
import com.sentiment.contentextractor.ContentExtractor;
import com.sentiment.contentextractor.News;
import com.sentiment.parser.news.ChinanewsNews;
import com.sentiment.parser.news.IfengNews;
import com.sentiment.parser.news.PeopleNews;
import com.sentiment.parser.news.QqNews;
import com.sentiment.parser.news.SinaNews;
import com.sentiment.parser.news.SohuNews;
import com.sentiment.parser.news.WangyiNews;
import com.sentiment.webcollector.crawler.DeepCrawler;
import com.sentiment.webcollector.model.Links;
import com.sentiment.webcollector.model.Page;

public class NewsCrawler extends DeepCrawler {
	private static final Logger LOG = LoggerFactory.getLogger(NewsCrawler.class);

	private List<News> newsList = new ArrayList<News>();

	public List<News> getNewsList() {
		return newsList;
	}

	public NewsCrawler(String crawlPath) {
		super(crawlPath);
		// TODO Auto-generated constructor stub
	}

	static {
		PropertyConfigurator.configure(Config.log4jPropertiesPath);
	}

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		NewsCrawler crawler = new NewsCrawler(Config.crawlerDataPath);
		crawler.getSearch("习近平+巴基斯坦+中巴");
		List<News> list = crawler.getNewsList();
		LOG.info(" 总共爬取的新闻数为 ： " + list.size());
	}

	@Override
	public Links visitAndGetNextLinks(Page page) {
		// TODO Auto-generated method stub
		String url = page.getUrl();
		Links links = new Links();
		if (url.matches(ConfigCrawler.toutiaoRegx)) {
			// PhantomjsCore phantomjsCore = new PhantomjsCore();
			// String html = phantomjsCore.parser(page);
			// page.setHtml(html);
			links.addAllFromDocument(page.getDoc(), "a[ga_event=feed_title]");
			return links;
		} else if (url.matches(ConfigCrawler.baiduRegx)) {
			links.addAllFromDocument(page.getDoc(), ".c-title");
			return links;
		} else if (url.matches(ConfigCrawler.youdaoRegx)) {
			links.addAllFromDocument(page.getDoc(), "#results");
			return links;
		} else {

			try {
				News news = ContentExtractor.getNewsByDoc(page.getDoc());
				if (news != null) {
					news.setUrl(url);
					synchronized (newsList) {
						newsList.add(news);
					}
				}
				// LOG.info(news.toString());
			} catch (Exception e) {
				LOG.error(e.toString());
			}
			return null;
		}
	}

	public void getSearch(String keyword) {
		try {
			keyword = URLEncoder.encode(keyword, "utf-8");
			// 今日头条搜索引擎
			String toutiaoStr = ConfigCrawler.toutiaoCrw + keyword;
			// 百度新闻搜索引擎
			String baiduStr = ConfigCrawler.baiduCrw + keyword;
			// 有道搜索
			String youdaoStr = ConfigCrawler.youdaoCrw + keyword;
			// 可以添加更多的搜索引擎
			addSeed(youdaoStr);
			addSeed(baiduStr);
			addSeed(toutiaoStr);

			// 搜索深度为2层
			start(2);

		} catch (Exception e) {
			LOG.error(e.toString());
		}
	}
}
